from datetime import datetime
import logging
import re
import pandas as pd
from rawdata_stored import save_raw_data_to_csv

class DataCleaner:
    def __init__(self):
        pass
    
    def clean(self, raw_posts):
        """数据清洗主方法（支持多种类型输入）"""
        # 处理不同类型输入
        if isinstance(raw_posts, pd.DataFrame):
            raw_posts = raw_posts.to_dict('records')
            logging.info("转换DataFrame为字典列表")
        
        if not isinstance(raw_posts, list):
            raise ValueError("raw_posts 必须是字典列表或DataFrame")
        
        cleaned_posts = []
        seen_ids = set()
        
        for post in raw_posts:
            try:
                # 确保 post 是字典
                if not isinstance(post, dict):
                    logging.warning(f"跳过非字典数据: {type(post)}")
                    continue
                
                # 文本清洗
                if 'text' in post:
                    post['text'] = self.clean_text(post['text'])
                else:
                    post['text'] = '[文本缺失]'
                
                # 缺失值处理
                if not post['text']:
                    post['text'] = '[内容已删除]'
                
                # 去重
                post_id = post.get('id')
                if post_id:
                    if post_id not in seen_ids:
                        seen_ids.add(post_id)
                        # 时间标准化
                        if 'timestamp' in post:
                            post['timestamp'] = self.standardize_time(post['timestamp'])
                        else:
                            post['timestamp'] = self.standardize_time(None)
                        cleaned_posts.append(post)
                else:
                    # 没有ID的帖子也处理
                    if 'timestamp' in post:
                        post['timestamp'] = self.standardize_time(post['timestamp'])
                    else:
                        post['timestamp'] = self.standardize_time(None)
                    cleaned_posts.append(post)
            except Exception as e:
                logging.warning(f"数据清洗失败: {str(e)}")
                continue
        
        return cleaned_posts
    
    def clean_text(self, text):
        """清洗文本内容"""
        if not text or not isinstance(text, str):
            return ''
        
        text = re.sub(r'<.*?>', '', text)  # 移除HTML标签
        text = re.sub(r'#\w+#', '', text)   # 移除话题标签
        text = re.sub(r'@[\w\u4e00-\u9fa5]+', '', text)  # 移除@用户
        text = re.sub(r'https?://\S+', '', text)  # 移除URL
        return text.strip()
    
    def standardize_time(self, raw_time):
        """时间标准化（更健壮的实现）"""
        if not raw_time:
            return datetime.now().isoformat()
        
        try:
            # 处理字符串
            if isinstance(raw_time, str):
                # 尝试多种格式
                for fmt in ('%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M', '%Y/%m/%d %H:%M'):
                    try:
                        dt = datetime.strptime(raw_time, fmt)
                        return dt.isoformat()
                    except:
                        continue
            
            # 处理时间戳
            if isinstance(raw_time, (int, float)):
                return datetime.fromtimestamp(raw_time).isoformat()
                
            return datetime.now().isoformat()
        except:
            return datetime.now().isoformat()

if __name__ == '__main__':
    cleaner = DataCleaner()
    raw_posts_path = r'D:\Users\Lenovo\Desktop\作业4\raw_data.csv'
    
    # 安全读取 CSV，处理各种异常情况
    try:
        # 处理各种编码问题
        for encoding in ['utf-8', 'gbk', 'latin1']:
            try:
                raw_posts = pd.read_csv(raw_posts_path, encoding=encoding)
                logging.info(f"成功使用 {encoding} 读取CSV")
                break
            except:
                continue
        
        # 转换为字典列表
        cleaned_posts = cleaner.clean(raw_posts)
        
        # 输出一些统计信息
        print(f"原始数据量: {len(raw_posts)}")
        print(f"清洗后数据量: {len(cleaned_posts)}")
        
        save_raw_data_to_csv(
            cleaned_posts,
            filename=r'D:\Users\Lenovo\Desktop\作业4\clean_data.csv'
        )
        
    except Exception as e:
        logging.error(f"数据处理失败: {str(e)}")
        # 创建简易备份
        with open('backup_data.txt', 'w') as f:
            f.write("原始数据路径: " + raw_posts_path + "\n")
            f.write(f"错误信息: {str(e)}")